#!/usr/bin/python
# coding: utf-8

from xml.dom.minidom import parse
from mensaje import mensaje
import re
import sys

from collocations import tTestGetCollocations, tTestReplaceCollocations

DEBUG_LEVEL=1


def raizPertenece(word, word_list):
    for a in word_list:
        raiz = word
        if len(word) > 3:
            if raiz.endswith('es'): raiz = raiz[:-2]
            if len(raiz) > 3 and raiz.endswith('s'): raiz = raiz[:-1]
            if len(raiz) > 3 and raiz.endswith('a'): raiz = raiz[:-1] + 'o'
        if a == word or a == raiz:
            return True
    
    return False


#
# Main
#


if len(sys.argv) > 1:


        # Notar que están incorporados las posibilidades luego de encontrar collocations
        candidatos = {
                u'cristina': [u'cfk',u'kristina',u'kirchner',u'klishtina',u'cristina_kirchner', u'cristina_fernandez', u'cristina_fernández', u'néstor_kirchner', u'nestor_kirchner', u'los_k'],
                u'lavagna':[u'sr lavagna', u'roberto_lavagna', u'dr_lavagna'],
                u'murphy':[u'bulldog', u'buldog', u'murfy', u'lópez_murphy', u'lopez_murphy', u'bull_dog'],
                u'sobisch': [u'bigoton', u'bigotón'],
                u'ripoll' :[u'vilma_ripoll'],
                u'solanas':[u'pino', u'pinosolanas', u'pino_solanas'],
                u'carrio': [u'carrio', u'gorda', u'la_gorda', u'sra_carrió', u'elisa_carrio', u'elisa_carrió'],
                u'pitrola' :[],
                u'saa': [u'adolfo', u'rodriguez_saa', u'rodríguez_saa', u'r_saa'],
                u'pitrola' :[]
        }
        
        # generate candidatosForms from previous dict of lists
        candidatosForms = {}
        for canonical in candidatos.iterkeys() :
                candidatosForms[canonical] = canonical
                for alternative in candidatos[canonical] : 
                        candidatosForms[alternative] = canonical





        inversores = set(['no','nunca','nada','ni','jamás','jamas'])
        pal_pos = []
        pal_neg = []
        
        fd_pos = open("dictionary/palabras_positivas.txt")
        for pal in fd_pos.readlines():
            pal_pos.append(pal.strip())
        fd_pos.close()

        fd_neg = open("dictionary/palabras_negativas.txt")
        for pal in fd_neg.readlines():
            pal_neg.append(pal.strip())
        fd_neg.close()

        if DEBUG_LEVEL > 0:
            print pal_pos
            print pal_neg


        #load corpus from command line input
	assert(sys.argv[1])
	fd = open(sys.argv[1], "r")					# open file

	corpus = {} # corpus is a dictionary post_id : list of post words
	m = mensaje()
	doc = parse(fd)
	for root in doc.childNodes :
		if root.localName=="corpus" :
			for message in root.childNodes :
				#print "DEBUG: ", message.localName
				if message.localName=="mensaje" :
					m.load_from_dom(message) # got the text, split into words
			                texto = re.sub("<br>|quot;|amp;|gt;|http://[^\s]*",'',m.texto)

                                        ws = map(unicode.lower, re.findall(u'[a-zA-Z0-9áéíóúÁÉÍÓÚñÑüÜ]+',unicode(texto)))
					#print "DEBUG:", ws
                                        
                                        corpus[m.id] = ws
                                        #print corpus[m.id]
	fd.close()



        replace = raw_input("Find and replace collocations? (y/n): ")
        no_negative = raw_input("Consider polarity inversors as negative words if no polarized word is found in the window? (y/n): ")

        
        if replace == 'y':
            #######find and replace collocations
            
            # Load all corpus from candidates
            corpus_candidate = {}
            for candidato in candidatos.iterkeys():
                print "DEBUG: Loading", candidato
                try:
                    fd = open("../corpus/paginas_institucionales/"+candidato+".txt", "r")	# open file
                except:
                    continue
                #corpus = [] # corpus is a list of words
                # All corpus are in latin-1
                corpus_candidate[candidato] = map(unicode.lower, re.findall(u'[a-zA-Z0-9áéíóúÁÉÍÓÚñÑüÜ]+',unicode(fd.read().decode('latin-1'))))
                fd.close()
                
            # Load both corpus
            corpus_files = ['../corpus/20071028_clarin_elecciones_presidenciales.txt','../corpus/20071024_psicofxp_elecciones_presidenciales.txt']
            
            #for corpus_file in corpus_files:
                #corpus_forum[corpus_file] = {}
            corpus_forum = {}
            
            for corpus_file in corpus_files:
                print "DEBUG: Loading ", corpus_file
                fd = open(corpus_file, "r") # open file
                corpus_forum[corpus_file] = {} # corpus is a dictionary post_id : list of post words
                m = mensaje()
                doc = parse(fd)
                for root in doc.childNodes :
                        if root.localName=="corpus" :
                                for message in root.childNodes :
                                        #print "DEBUG: ", message.localName
                                        if message.localName=="mensaje" :
                                                m.load_from_dom(message) # got the text, split into words
                                                texto = re.sub("<br>|quot;|amp;|gt;|http://[^\s]*",'',m.texto)
                                                ws = map(unicode.lower, re.findall(u'[a-zA-Z0-9áéíóúÁÉÍÓÚñÑüÜ]+',unicode(texto)))
                                                                #print "DEBUG:", ws
                                                corpus_forum[corpus_file][m.id] = ws
                                                #print corpus_forum[m.id]
                fd.close()
            
            
            # Accumulate all corpora to get collocations globally
            all_corpus = []
            for c in corpus_candidate.iterkeys() :
                    all_corpus.extend(corpus_candidate[c])
            for corpus_file in corpus_files:
                for i in corpus_forum[corpus_file].iterkeys() :
                        all_corpus.extend(corpus_forum[corpus_file][i])
            # Generate collocations from all corpora. Computation INTENSIVE!!!
            print "DEBUG: generating collocations"
            collocations = tTestGetCollocations(all_corpus, topk=16384, topj=512)
            #print "DEBUG: collocations", collocations
            print "DEBUG:replacing collocations"
            for c in corpus_candidate.iterkeys() :
                    corpus_candidate[c] = tTestReplaceCollocations(corpus_candidate[c], collocations)
            for corpus_file in corpus_files:
                for i in corpus_forum[corpus_file].iterkeys() :
                        corpus_forum[corpus_file][i] = tTestReplaceCollocations(corpus_forum[corpus_file][i], collocations)
            all_corpus = [] #please GC all this
            
            
            #######end find and replace collocations




        w_size = 10
        
        
        resultados = {}
        for k in candidatos.keys():
            resultados[k] = {}
        
        windows = {}
        
        for (post_id, post_words) in corpus.iteritems():
            for i in range(len(post_words)):
                if post_words[i] in candidatosForms.keys():
                    start = max(0, i - w_size/2)
                    end   = min(i + w_size/2 + 1, len(corpus))
                    
                    
                    if not windows.has_key(post_id):
                        windows[post_id] = [(candidatosForms[post_words[i]].decode('latin-1'),post_words[start:end])]
                    else:
                        windows[post_id].append((candidatosForms[post_words[i]].decode('latin-1'),post_words[start:end]))
                    
        for post_id, l in windows.items(): # l = lista de (post_id, (candidato, ventana))
            for candidato,palabras in l: #l = (candidato,ventana)
            
                #palabras = [result[1] for result in freeling_result[l[2][0]:l[2][1]] if result[1].decode('latin-1') not in  candidatosForms.keys()]
                positivas,negativas,sinclasif = [],[],[]
                
                if set(palabras).intersection(inversores) == set():
                    #no ocurre ningun inversor de polaridad
                    for a in palabras:
                        if a.encode('latin-1') in pal_pos or raizPertenece(a.encode('latin-1'),pal_pos):
                            positivas.append(a)
                        elif a.encode('latin-1') in pal_neg or raizPertenece(a.encode('latin-1'),pal_neg):
                            negativas.append(a)
                        else:
                            sinclasif.append(a)
    
                else:
                    #invertir
                    for a in palabras:
                        if a.encode('latin-1') in pal_neg or raizPertenece(a.encode('latin-1'),pal_neg):
                            positivas.append(a)
                        elif a.encode('latin-1') in pal_pos or raizPertenece(a.encode('latin-1'),pal_pos):
                            negativas.append(a)
                        else:
                            sinclasif.append(a)
    
                    if no_negative == 'y' and not positivas and not negativas:
                        #solo se encontraron inversores pero ninguna palabra buena o mala
                        #entonces las considero palabra mala
                        negativas.extend(list(set(palabras).intersection(inversores)))
    
    
                    
                if positivas or negativas:
                    #print "para post=",
                    #print post_id
                    #print "candidato=",
                    #print candidato
                    #print "palabras encontradas:"
                    #print "positivas=",
                    #print positivas
                    #print "negativas=",
                    #print negativas
                    #print "sin clasificar=",
                    #print sinclasif
                    #print "de=",
                    #print palabras
                    if resultados[candidato].has_key(post_id):
                        resultados[candidato][post_id][0] += len(positivas)
                        resultados[candidato][post_id][1] += len(negativas)
                    else:
                        resultados[candidato][post_id] = [len(positivas),len(negativas)]



        #print resultados
        
        #Acumulo la cantidad de posts positivos y de posts negativos por cada
        posts_classifications = {}
        for candidato in candidatos.iterkeys():
            posts_classifications[candidato] = (0,0)
        
        for candidato in candidatos.iterkeys():
            cant_positivos = 0
            cant_negativos = 0
            for post in resultados[candidato].iteritems():
                if post[1][0] > post[1][1]:
                    if DEBUG_LEVEL > 0:
                        print "post",
                        print post[0],
                        print "marcado como positivo"
                    cant_positivos += 1
                elif post[1][0] < post[1][1]:
                    if DEBUG_LEVEL > 0:
                        print "post",
                        print post[0],
                        print "marcado como negativo"
                    cant_negativos += 1
            print cant_positivos,
            print "posts positivos y",
            print cant_negativos,
            print "posts negativos para",
            print candidato
            posts_classifications[candidato] = (cant_positivos,cant_negativos)
        
        posts_ratio = {}
        for candidato in posts_classifications.iterkeys():
            if posts_classifications[candidato][1] == 0.0:
                if posts_classifications[candidato][0] == 0.0:
                    posts_ratio[candidato] = 0.0
                else:
                    posts_ratio[candidato] = 9999999999999.9
            else:
                posts_ratio[candidato] = float(posts_classifications[candidato][0]) / float(posts_classifications[candidato][1])
        
        l = posts_ratio.items()
	l.sort(lambda x,y : cmp(x[1],y[1]), reverse=True)
        print l
        print l[0][0], "presidente!"
        
        inp = True
        while inp:
            inp = raw_input("Ver el contenido de algun post: ")
            if not inp:
                break
            texto = " ".join(corpus[inp])
            texto = re.sub("("+"|".join(candidatosForms.keys())+")",r"__\1__",texto)
            print(texto)

        
        sys.exit(0)

else:
	print "Usage:", sys.argv[0], "corpus.xml"
	print
	print "Example:", sys.argv[0], "clarin.xml"

